/**
 *  财经--财经新闻
 *  http://www.chinanews.com/cj/gd.shtml
 */

var cheerio = require("cheerio");//引用抓取html结构的node模块
var mkdirp  = require("mkdirp");  //引用创建文件夹的node模块
var iconv   = require("iconv-lite"); //引用字符集node模块
var commen  = require("../allocation/commen");//引用公共操作的自定义文件
var data    = require("../allocation/sql");  //引用数据库操作的自定义文件
var async   = require("async");  //引用同步执行

var url       = "http://www.chinanews.com/cj/gd.shtml";
var chanel    = "caijingxinwen";
var newsType  = 10;
var imgSite   = "http://www.chinanews.com";
var website   = "http://www.chinanews.com";
var newsLists = [];   //存放新闻列表
var logs      = [];   //存放日志信息



//创建存放图片的文件夹及路径(20160901)
var imgPath               = commen.createDownFile().imgPath;

mkdirp.sync(imgPath);
logs.push({
    start_time:new Date().toLocaleString(),
    article_type_id:newsType,
    thread:chanel
});

//定义获取新闻列表
function getNewsList(url,callback)
{
    commen.load(url,function(err,body){
        if(err)
        {
            logs[0].location = "列表请求";
            logs[0].message  = "错误："+err;
            logs[0].level    = 0;
            logs[0].end_time = new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("列表请求出错了！！！!");
                    process.exit(0);
                }
            });
            return err;
        }
        var  $  =cheerio.load(body.toString());
        var articleList = [];
        //获取新闻列表
        var i = 0;
        $(".dd_bt").each(function () {
            if(i<21){
                var sear = new RegExp('http');
            　　if(sear.test($(this).find("a").attr("href")))
            　　{
                    //articleList.push($(this).find("a").attr("href"));
                }
                else{
                    articleList.push(website+$(this).find("a").attr("href"));
                }
            }
            i++;
            
        });
        //console.log(articleList);
        callback(null, articleList);
    });
}

//定义获取新闻内容
function getNewsContent(url,callback)
{
    console.log(url);
    commen.load(url,function(err,body){
        if(err)
        {
            logs[0].location = "内容请求";
            logs[0].message  = "错误："+err;
            logs[0].level    = 0;
            logs[0].end_time = new Date().toLocaleString();
            data.insertLog(logs[0],function(err,result){
                if(err) throw err;
                if(result>0)
                {
                    console.log("内容请求出错了！！！");
                    process.exit(0);
                }
            });
            return err;
        }
        var html        = iconv.decode(body, 'gb2312');
        var $           = cheerio.load(html.toString(), {decodeEntities: false});
        var news        = [];
        var picUrls     = []; //存放图片的数组
        var allContent  = "";  //存放排版好的ubb格式的内容
        if($(".left_zw   img").length>0)
            {
                for(var i=0;i<$(".left_zw  img").length;i++)
                {
                    if($($(".left_zw img")[i]).attr("src").indexOf("title")>0)
                {
                    $($(".left_zw img")[i]).remove();
                    continue;
                }
                    //改
                    picUrls.push($($(".left_zw   img")[i]).attr("src"));
                    $($(".left_zw   img")[i]).removeAttr("title");
                    $($(".left_zw   img")[i]).removeAttr("alt");
                    $($(".left_zw   img")[i]).html("[text-align:center]"+$($(".left_zw   img")[i]).attr("src")+"[/img]");
                    $($(".left_zw   img")[i]).removeAttr("src");
                }
            }

         $(".left_zw script").remove();
            $(".left_zw").html($(".left_zw").html().replace(/<!--[\w\W\r\n]*?-->/gmi,'').trim());
            $(".left_zw").html($(".left_zw").html().replace(/<([a-zA-Z1-6]+)(\s*[^>]*)?>/g, "<$1>"));
            //找到内容中所有的p标签
            for (var i = 0; i <$(".left_zw p").length; i++) {
                if($($(".left_zw p")[i]).find("img")=="")
                {
                    $($(".left_zw p")[i]).text("[&emsp][&emsp]" + $($(".left_zw p")[i]).text().trim());
                }
                allContent += $($(".left_zw p")[i]).text();
            }
            allContent = $(".left_zw").html().trim().replace(/>/g, "]").replace(/</g, "[").replace(/&rdquo;/g, "[&rdquo]").replace(/&ldquo;/g,"[&ldquo]").replace(/&nbsp;/g,"[&nbsp]").replace(/&amp;/g,"[&amp]").replace(/&gt;/g,"[&gt]").replace(/&lt;/g,"[&lt]");
        //将插入数据库的值存入数组news中
        news.push({
            title   : $("#cont_1_1_2 > h1").text().trim(),
            source  : "中国新闻网",
            author  : "wangluo",
            date    : new Date().toLocaleString(),
            typeID  : newsType,
            chanel  : chanel,
            content : allContent,
            picUrls : picUrls
        });
        
        
        callback(null, news[0]);
    });
}

//调用方法，依次获取列表和内容，并存入数据库
async.series([
    // 获取新闻列表
    function (done) {
        getNewsList(url,function(err,list){
            newsLists = list;
            done(err);
        });
    },
    function (done) {
        async.eachSeries(newsLists, function (item, next) {
            getNewsContent(item, function (err,detail) {
                if (err) console.error(err.stack);
                data.insertData(detail,imgPath,function(err,result){
                   // console.log(detail);  //数组(title,source等，打印内容，就是detail.content)
                    if(err)
                    {
                        logs[0].location = "入库";
                        logs[0].message  = "错误："+err;
                        logs[0].level    = 0;
                        logs[0].end_time = new Date().toLocaleString();
                        data.insertLog(logs[0],function(err,result){
                            if(err) throw err;
                            if(result>0)
                            {
                                console.log("入库出错了！！！");
                                process.exit(0);
                            }
                        });
                        return err;
                    }
                    else if(result.affectedRows>0)
                     {
                         console.log("采集成功");
                     }
                });
                next();
            });
        }, done);
    }
], function (err) {
    if (err) console.error(err.stack);
    logs[0].location = "无";
    logs[0].message  = "正确";
    logs[0].level    = 1;
    logs[0].end_time = new Date().toLocaleString();
    data.insertLog(logs[0],function(err,result){
     if(err) throw err;
     if(result>0)
      {
         console.log("写入日志成功!");
          console.log('采集完成');
         process.exit(0);
      }
     });
});



















